import pandas as pd


# plotting
import seaborn as sn
import matplotlib.pyplot as plt

# 读入数据
train = pd.read_csv("train.csv",nrows=10000000)
print("Train dataset dimensions:", train.shape)
print("Train dataset info:", train.info())
# 查找有无缺失值
print("Train dataset null:", train.isnull().any())


# 正负例样本情况
print('正负样本情况', train['click'].value_counts())


Cat_features = train.columns.drop(['id','click'])

#查看所有类别型特征的分布，train都可以看成是类别型的
for col in Cat_features:
    num_vlaules = len(train[col].unique())
    print ('\n%s属性有%d的不同取值，各取值及其出现的次数\n'% (col,num_vlaules) )
    print (train[col].value_counts())

train['hour_time']=train['hour'].astype(str)
train['hour_time']=[temp[-2:] for temp in train['hour_time']]

Cat_features_MEncoder = ['site_id','site_domain','app_id','app_domain','device_id','device_ip','device_model','C14','C17','C20']
Cat_features_OnehotEncoder = ['C1','banner_pos','site_category','app_category','device_conn_type','C15','C16','C19','C21']

rare_thresholds = [100000,2000000,10000,30000,100000,20000,20000,10000,10000]
Cat_features_others = ['hour_time','device_type','C18']

#合并样本
j = 0
for col in Cat_features_OnehotEncoder:
    # 每个取值的样本数目
    train[col] = train[col].astype(str)
    value_counts_col = train[col].value_counts(dropna=False)
    threshold = rare_thresholds[j]

    # 样本数目小于阈值的取值为稀有取值
    value_counts_rare = list(value_counts_col[value_counts_col < threshold].index)

    # 稀有值合并为：others
    rare_index = train[col].isin(value_counts_rare)
    # print(rare_index)
    train[col].loc[rare_index] = "Others"
    j = j + 1

#查看所有类别型特征的分布，train都可以看成是类别型的
for col in Cat_features:
    num_vlaules = len(train[col].unique())
    print ('\n%s属性有%d的不同取值，各取值及其出现的次数\n'% (col,num_vlaules) )
    print (train[col].value_counts())

train.to_csv('train_new.csv',index=False)